{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T09:56:00.173279Z",
     "start_time": "2019-10-13T09:56:00.169655Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt \n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import catboost as cb\n",
    "import lightgbm as lgb\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV, StratifiedKFold,TimeSeriesSplit,KFold,GroupKFold\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import sqlite3\n",
    "import xgboost as xgb\n",
    "import datetime\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from scipy.stats import pearsonr\n",
    "import gc\n",
    "from sklearn.model_selection import TimeSeriesSplit\n",
    "import hashlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "__print__ = print\n",
    "def print(string):\n",
    "    __print__(string)\n",
    "    os.system(f'echo \\\"{string}\\\"')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 读取数并进行预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T09:56:00.607892Z",
     "start_time": "2019-10-13T09:56:00.600932Z"
    }
   },
   "outputs": [],
   "source": [
    "# 对DeviceInfo，id_30，id_31进行处理，并生成一个是否有identity的特征\n",
    "\n",
    "def id_split(df):\n",
    "    df['device_name'] = df['DeviceInfo'].str.split('/', expand=True)[0]\n",
    "    df['device_version'] = df['DeviceInfo'].str.split('/', expand=True)[1]\n",
    "\n",
    "    df['OS_id_30'] = df['id_30'].str.split(' ', expand=True)[0]\n",
    "\n",
    "    df['browser_id_31'] = df['id_31'].str.split(' ', expand=True)[0]\n",
    "\n",
    "    df.loc[df['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'\n",
    "    df.loc[df['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'\n",
    "    df.loc[df['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'\n",
    "    df.loc[df['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'\n",
    "    df.loc[df['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'\n",
    "    df.loc[df['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'\n",
    "    df.loc[df['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'\n",
    "    df.loc[df['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'\n",
    "    df.loc[df['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'\n",
    "    df.loc[df['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'\n",
    "    df.loc[df['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'\n",
    "    df.loc[df['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'\n",
    "    df.loc[df['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'\n",
    "    df.loc[df['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'\n",
    "    df.loc[df['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'\n",
    "    df.loc[df['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'\n",
    "    df.loc[df['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'\n",
    "    \n",
    "    # 类别太稀疏的置为其他类    \n",
    "    df.loc[df.device_name.isin(df.device_name.value_counts()[df.device_name.value_counts() < 100].index), 'device_name'] = \"Others\"\n",
    "    df['had_id'] = 1\n",
    "    gc.collect()\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T09:48:16.817429Z",
     "start_time": "2019-10-13T09:47:51.974589Z"
    }
   },
   "outputs": [],
   "source": [
    "train_transaction = pd.read_csv('../input/ieee-fraud-detection/train_transaction.csv')\n",
    "train_identity =    pd.read_csv('../input/ieee-fraud-detection/train_identity.csv')\n",
    "test_transaction =  pd.read_csv('../input/ieee-fraud-detection/test_transaction.csv')\n",
    "test_identity =     pd.read_csv('../input/ieee-fraud-detection/test_identity.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T09:55:49.71555Z",
     "start_time": "2019-10-13T09:55:49.712586Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(590540, 394)\n",
      "(144233, 41)\n",
      "(506691, 393)\n",
      "(141907, 41)\n"
     ]
    }
   ],
   "source": [
    "print(train_transaction.shape)\n",
    "print(train_identity.shape)\n",
    "print(test_transaction.shape)\n",
    "print(test_identity.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['TransactionID', 'id_01', 'id_02', 'id_03', 'id_04', 'id_05', 'id_06',\n",
       "       'id_07', 'id_08', 'id_09', 'id_10', 'id_11', 'id_12', 'id_13', 'id_14',\n",
       "       'id_15', 'id_16', 'id_17', 'id_18', 'id_19', 'id_20', 'id_21', 'id_22',\n",
       "       'id_23', 'id_24', 'id_25', 'id_26', 'id_27', 'id_28', 'id_29', 'id_30',\n",
       "       'id_31', 'id_32', 'id_33', 'id_34', 'id_35', 'id_36', 'id_37', 'id_38',\n",
       "       'DeviceType', 'DeviceInfo'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_identity.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['TransactionID', 'id-01', 'id-02', 'id-03', 'id-04', 'id-05',\n",
       "       'id-06', 'id-07', 'id-08', 'id-09', 'id-10', 'id-11', 'id-12',\n",
       "       'id-13', 'id-14', 'id-15', 'id-16', 'id-17', 'id-18', 'id-19',\n",
       "       'id-20', 'id-21', 'id-22', 'id-23', 'id-24', 'id-25', 'id-26',\n",
       "       'id-27', 'id-28', 'id-29', 'id-30', 'id-31', 'id-32', 'id-33',\n",
       "       'id-34', 'id-35', 'id-36', 'id-37', 'id-38', 'DeviceType',\n",
       "       'DeviceInfo'], dtype=object)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_identity.columns.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "col1 = ['TransactionID']\n",
    "col2 = ['id_0'+str(n) for n in np.arange(1, 10)]\n",
    "col3 = ['id_'+str(n) for n in np.arange(10, 39)]\n",
    "col4 = ['DeviceType', 'DeviceInfo']\n",
    "col =  col1 + col2 + col3 + col4\n",
    "# col\n",
    "test_identity.columns = col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T09:56:09.202955Z",
     "start_time": "2019-10-13T09:56:04.656447Z"
    }
   },
   "outputs": [],
   "source": [
    "train_identity = id_split(train_identity)\n",
    "test_identity = id_split(test_identity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T09:56:13.643065Z",
     "start_time": "2019-10-13T09:56:13.640269Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(144233, 46)\n",
      "(141907, 46)\n"
     ]
    }
   ],
   "source": [
    "print(train_identity.shape)\n",
    "print(test_identity.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T10:25:54.204327Z",
     "start_time": "2019-10-13T10:25:54.152345Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Windows            47722\n",
      "iOS Device         19782\n",
      "MacOS              12573\n",
      "Samsung            12092\n",
      "Trident             7440\n",
      "Others              4978\n",
      "RV                  4385\n",
      "Motorola            2935\n",
      "Huawei              2377\n",
      "LG                  2331\n",
      "Sony                 575\n",
      "ZTE                  518\n",
      "HTC                  406\n",
      "hi6210sft Build      190\n",
      "F3213 Build          125\n",
      "Linux                121\n",
      "F5121 Build          116\n",
      "Name: device_name, dtype: int64\n",
      "7.0              7440\n",
      "NRD90M           5908\n",
      "MMB29K           1874\n",
      "MRA58K           1446\n",
      "MMB29M           1342\n",
      "                 ... \n",
      "HUAWEILYO-L21       1\n",
      "NJH47F              1\n",
      "34.2.A.2.47         1\n",
      "Q1010               1\n",
      "V41020c             1\n",
      "Name: device_version, Length: 293, dtype: int64\n",
      "Windows    36739\n",
      "iOS        19782\n",
      "Mac        13580\n",
      "Android     6303\n",
      "Linux       1136\n",
      "other         15\n",
      "func          10\n",
      "Name: OS_id_30, dtype: int64\n",
      "chrome               76059\n",
      "mobile               28379\n",
      "ie                    9733\n",
      "safari                8913\n",
      "firefox               7012\n",
      "edge                  6401\n",
      "samsung               2044\n",
      "opera                  449\n",
      "android                386\n",
      "other                  312\n",
      "Samsung/SM-G532M       150\n",
      "google                 146\n",
      "Generic/Android        138\n",
      "Samsung/SM-G531H        52\n",
      "Microsoft/Windows       25\n",
      "silk                    19\n",
      "ZTE/Blade                9\n",
      "comodo                   6\n",
      "line                     6\n",
      "maxthon                  6\n",
      "Mozilla/Firefox          5\n",
      "icedragon                5\n",
      "aol                      5\n",
      "Lanix/Ilium              3\n",
      "waterfox                 2\n",
      "palemoon                 2\n",
      "facebook                 2\n",
      "puffin                   2\n",
      "Inco/Minion              1\n",
      "Nokia/Lumia              1\n",
      "iron                     1\n",
      "Samsung/SCH              1\n",
      "BLU/Dash                 1\n",
      "LG/K-200                 1\n",
      "M4Tel/M4                 1\n",
      "Cherry                   1\n",
      "seamonkey                1\n",
      "cyberfox                 1\n",
      "chromium                 1\n",
      "Name: browser_id_31, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# 新增了5个字段\n",
    "print(train_identity.device_name.value_counts())\n",
    "print(train_identity.device_version.value_counts())\n",
    "print(train_identity.OS_id_30.value_counts())\n",
    "print(train_identity.browser_id_31.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T10:28:31.248236Z",
     "start_time": "2019-10-13T10:28:20.083165Z"
    }
   },
   "outputs": [],
   "source": [
    "train = pd.merge(train_transaction, train_identity, on = 'TransactionID', how = 'left')\n",
    "test = pd.merge(test_transaction, test_identity, on = 'TransactionID', how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T10:34:26.097821Z",
     "start_time": "2019-10-13T10:34:20.36484Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5447467732477208009266664\n",
      "4667966862746899078655461\n"
     ]
    }
   ],
   "source": [
    "print(sum(pd.util.hash_pandas_object(train)))\n",
    "#5447467732477208009266664\n",
    "print(sum(pd.util.hash_pandas_object(test)))\n",
    "#4667966862746899078655461"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T10:40:59.250368Z",
     "start_time": "2019-10-13T10:40:59.213087Z"
    }
   },
   "outputs": [],
   "source": [
    "del train_transaction\n",
    "del test_transaction\n",
    "del train_identity\n",
    "del test_identity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train = train[:5000]\n",
    "# test = test[:5000]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 特征工程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_len = len(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T10:42:39.909515Z",
     "start_time": "2019-10-13T10:42:38.390467Z"
    }
   },
   "outputs": [],
   "source": [
    "# 加入DT_D  第几天，\n",
    "#    DT_W  第几周，\n",
    "#    DT_M  第几月\n",
    "START_DATE = '2017-11-30'\n",
    "startdate = datetime.datetime.strptime(START_DATE, '%Y-%m-%d')\n",
    "train['TransactionDT'] = train['TransactionDT'].apply(lambda x: (startdate + datetime.timedelta(seconds = x)))\n",
    "test['TransactionDT'] = test['TransactionDT'].apply(lambda x: (startdate + datetime.timedelta(seconds = x)))\n",
    "for df in [train,test]:\n",
    "    df['DT_D'] = ((df['TransactionDT'].dt.year-2017)*365 + df['TransactionDT'].dt.dayofyear).astype(np.int16)\n",
    "    df['DT_W'] = (df['TransactionDT'].dt.year-2017)*52 + df['TransactionDT'].dt.weekofyear\n",
    "    df['DT_M'] = (df['TransactionDT'].dt.year-2017)*12 + df['TransactionDT'].dt.month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T11:17:17.088019Z",
     "start_time": "2019-10-13T11:17:17.050332Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "W    439670\n",
       "C     68519\n",
       "R     37699\n",
       "H     33024\n",
       "S     11628\n",
       "Name: ProductCD, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.ProductCD.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    因为意识到ProductCD各个种类在欺诈和时间序列上的表现差异很大，所以将他们的count_encoding拆解为5个指标"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T11:17:32.870267Z",
     "start_time": "2019-10-13T11:17:31.181947Z"
    }
   },
   "outputs": [],
   "source": [
    "#### R\n",
    "te = train.groupby(['ProductCD','DT_D'])['isFraud'].agg(['count','mean'])\n",
    "te.reset_index(inplace=True)\n",
    "train['ProductCD_R_Day'] = pd.merge(train[['ProductCD','DT_D']],te[['ProductCD','DT_D','count']],on = ['ProductCD','DT_D'],how='left')['count']\n",
    "te = test.groupby(['ProductCD','DT_D'])['TransactionAmt'].agg(['count','mean'])\n",
    "te.reset_index(inplace=True)\n",
    "test['ProductCD_R_Day'] = pd.merge(test[['ProductCD','DT_D']],te[['ProductCD','DT_D','count']],on = ['ProductCD','DT_D'],how='left')['count']\n",
    "train.loc[train.ProductCD != 'R','ProductCD_R_Day'] = -999\n",
    "test.loc[test.ProductCD != 'R','ProductCD_R_Day'] = -999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T11:17:33.178159Z",
     "start_time": "2019-10-13T11:17:32.871813Z"
    }
   },
   "outputs": [],
   "source": [
    "#### H\n",
    "te = train.groupby(['ProductCD','DT_D'])['isFraud'].agg(['count','mean'])\n",
    "te.reset_index(inplace=True)\n",
    "train['ProductCD_H_Day'] = pd.merge(train[['ProductCD','DT_D']],te[['ProductCD','DT_D','count']],on = ['ProductCD','DT_D'],how='left')['count']\n",
    "te = test.groupby(['ProductCD','DT_D'])['TransactionAmt'].agg(['count','mean'])\n",
    "te.reset_index(inplace=True)\n",
    "test['ProductCD_H_Day'] = pd.merge(test[['ProductCD','DT_D']],te[['ProductCD','DT_D','count']],on = ['ProductCD','DT_D'],how='left')['count']\n",
    "train.loc[train.ProductCD != 'H','ProductCD_H_Day'] = -999\n",
    "test.loc[test.ProductCD != 'H','ProductCD_H_Day'] = -999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T11:17:33.486022Z",
     "start_time": "2019-10-13T11:17:33.179651Z"
    }
   },
   "outputs": [],
   "source": [
    "#### C\n",
    "te = train.groupby(['ProductCD','DT_D'])['isFraud'].agg(['count','mean'])\n",
    "te.reset_index(inplace=True)\n",
    "train['ProductCD_C_Day'] = pd.merge(train[['ProductCD','DT_D']],te[['ProductCD','DT_D','count']],on = ['ProductCD','DT_D'],how='left')['count']\n",
    "te = test.groupby(['ProductCD','DT_D'])['TransactionAmt'].agg(['count','mean'])\n",
    "te.reset_index(inplace=True)\n",
    "test['ProductCD_C_Day'] = pd.merge(test[['ProductCD','DT_D']],te[['ProductCD','DT_D','count']],on = ['ProductCD','DT_D'],how='left')['count']\n",
    "train.loc[train.ProductCD != 'C','ProductCD_C_Day'] = 999999\n",
    "test.loc[test.ProductCD != 'C','ProductCD_C_Day'] = 999999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T11:17:33.79711Z",
     "start_time": "2019-10-13T11:17:33.487528Z"
    }
   },
   "outputs": [],
   "source": [
    "#### W\n",
    "te = train.groupby(['ProductCD','DT_D'])['isFraud'].agg(['count','mean'])\n",
    "te.reset_index(inplace=True)\n",
    "train['ProductCD_W_Day'] = pd.merge(train[['ProductCD','DT_D']],te[['ProductCD','DT_D','count']],on = ['ProductCD','DT_D'],how='left')['count']\n",
    "te = test.groupby(['ProductCD','DT_D'])['TransactionAmt'].agg(['count','mean'])\n",
    "te.reset_index(inplace=True)\n",
    "test['ProductCD_W_Day'] = pd.merge(test[['ProductCD','DT_D']],te[['ProductCD','DT_D','count']],on = ['ProductCD','DT_D'],how='left')['count']\n",
    "train.loc[train.ProductCD != 'W','ProductCD_W_Day'] = -999\n",
    "test.loc[test.ProductCD != 'W','ProductCD_W_Day'] = -999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T11:17:34.113308Z",
     "start_time": "2019-10-13T11:17:33.798572Z"
    }
   },
   "outputs": [],
   "source": [
    "#### S\n",
    "te = train.groupby(['ProductCD','DT_D'])['isFraud'].agg(['count','mean'])\n",
    "te.reset_index(inplace=True)\n",
    "train['ProductCD_S_Day'] = pd.merge(train[['ProductCD','DT_D']],te[['ProductCD','DT_D','count']],on = ['ProductCD','DT_D'],how='left')['count']\n",
    "te = test.groupby(['ProductCD','DT_D'])['TransactionAmt'].agg(['count','mean'])\n",
    "te.reset_index(inplace=True)\n",
    "test['ProductCD_S_Day'] = pd.merge(test[['ProductCD','DT_D']],te[['ProductCD','DT_D','count']],on = ['ProductCD','DT_D'],how='left')['count']\n",
    "train.loc[train.ProductCD != 'S','ProductCD_S_Day'] = -999\n",
    "test.loc[test.ProductCD != 'S','ProductCD_S_Day'] = -999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T11:18:40.885697Z",
     "start_time": "2019-10-13T11:18:40.88306Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ProductCD_R_Day', 'ProductCD_H_Day', 'ProductCD_C_Day', 'ProductCD_W_Day', 'ProductCD_S_Day']\n"
     ]
    }
   ],
   "source": [
    "# 新增了5个特征\n",
    "print(train.columns[-5:].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T11:23:29.458605Z",
     "start_time": "2019-10-13T11:23:24.769586Z"
    }
   },
   "outputs": [],
   "source": [
    "# 使用card系列和open_card的字段作为标识唯一用户的id\n",
    "train['open_card'] = train.DT_D - train.D1\n",
    "train['first_tran'] = train.DT_D - train.D2\n",
    "test['open_card'] = test.DT_D - test.D1\n",
    "test['first_tran'] = test.DT_D - test.D2\n",
    "\n",
    "train['uid1'] = train.card1.astype(str) +' '+ train.card2.astype(str)+' '+ train.card3.astype(str)+' '+train.card4.astype(str)+' '+ train.card5.astype(str)+' '+ train.card6.astype(str) +' '+ train.addr1.astype(str)+' '+train.addr2.astype(str)+' '+train.open_card.astype(str)\n",
    "test['uid1'] = test.card1.astype(str) +' '+ test.card2.astype(str)+' '+ test.card3.astype(str)+' '+ test.card4.astype(str)+' '+ test.card5.astype(str)+' '+ test.card6.astype(str) +' '+ test.addr1.astype(str)+' '+test.addr2.astype(str)+' '+test.open_card.astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T11:23:52.313428Z",
     "start_time": "2019-10-13T11:23:52.079967Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "222518\n",
      "198011\n"
     ]
    }
   ],
   "source": [
    "print(train['uid1'].nunique())\n",
    "print(test['uid1'].nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T11:46:41.226297Z",
     "start_time": "2019-10-13T11:46:41.223527Z"
    }
   },
   "outputs": [],
   "source": [
    "# 使用id_30，id_31，id_32，id_33，DeviceType，DeviceInfo标识一个设备\n",
    "def device_hash(x):\n",
    "    s =  str(x['id_30'])+str(x['id_31'])+str(x['id_32'])+str(x['id_33'])+str( x['DeviceType'])+ str(x['DeviceInfo'])\n",
    "    h = hashlib.sha256(s.encode('utf-8')).hexdigest()[0:15]\n",
    "    return h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T11:48:00.747092Z",
     "start_time": "2019-10-13T11:46:43.606149Z"
    }
   },
   "outputs": [],
   "source": [
    "for df in [train,test]:\n",
    "    df['device_hash'] = df.apply(lambda x: device_hash(x), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T11:49:06.126392Z",
     "start_time": "2019-10-13T11:49:03.630654Z"
    }
   },
   "outputs": [],
   "source": [
    "# 同uid设备的个数\n",
    "concat_df = pd.concat([train[['uid1','device_hash']],test[['uid1','device_hash']]])\n",
    "tmp = concat_df.groupby('uid1')['device_hash'].agg(['nunique'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T11:49:39.650527Z",
     "start_time": "2019-10-13T11:49:37.837253Z"
    }
   },
   "outputs": [],
   "source": [
    "train['uid_device_nunique'] = train.uid1.map(tmp.to_dict()['nunique'])\n",
    "test['uid_device_nunique'] = test.uid1.map(tmp.to_dict()['nunique'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T11:50:30.107548Z",
     "start_time": "2019-10-13T11:50:27.813946Z"
    }
   },
   "outputs": [],
   "source": [
    "# 同设备uid的个数\n",
    "tmp = concat_df.groupby('device_hash')['uid1'].agg(['nunique'])\n",
    "train['device_uid_nunique'] = train.device_hash.map(tmp.to_dict()['nunique'])\n",
    "test['device_uid_nunique'] = test.device_hash.map(tmp.to_dict()['nunique'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T11:51:50.465555Z",
     "start_time": "2019-10-13T11:51:50.446959Z"
    }
   },
   "outputs": [],
   "source": [
    "del concat_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T12:04:58.073721Z",
     "start_time": "2019-10-13T12:04:35.689373Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 得到金额不为零小数位的个数\n",
    "def change(hoge):\n",
    "    hoge = np.round(hoge,3)\n",
    "    num = 3\n",
    "    hoge = int(np.round(np.round(hoge,3)*1000))\n",
    "    while(hoge % 10 ==0):\n",
    "        num = num-1\n",
    "        hoge = hoge /10\n",
    "    if num<0:\n",
    "        num = 0\n",
    "    return num\n",
    "  \n",
    "train['decimal_digit'] = train[\"TransactionAmt\"].map(change)\n",
    "test['decimal_digit'] = test['TransactionAmt'].map(change)\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T12:06:02.601199Z",
     "start_time": "2019-10-13T12:06:02.588604Z"
    }
   },
   "outputs": [],
   "source": [
    "#没有identity填充为0\n",
    "train.had_id = train.had_id.fillna(0)\n",
    "test.had_id = test.had_id.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T12:13:37.637359Z",
     "start_time": "2019-10-13T12:13:02.115915Z"
    }
   },
   "outputs": [],
   "source": [
    "### scale\n",
    "### D系列数据有随时间增加的趋势，未来的数据大于过去的数据，所以进行缩放，将相对关系保留\n",
    "for t in ['D1','D2','D4','D6','D10','D11','D12','D14','D15']:\n",
    "    train[t+'_revised'] = train[t]/train.groupby('DT_W')[t].transform('max')\n",
    "    test[t+'_revised'] = test[t]/test.groupby('DT_W')[t].transform('max')\n",
    "for t in ['D3','D5','D7','D8','D13']:\n",
    "    train[t+'_revised'] = train[t]/train.groupby('DT_M')[t].transform('max')\n",
    "    test[t+'_revised'] = test[t]/test.groupby('DT_M')[t].transform('max')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T12:17:43.184232Z",
     "start_time": "2019-10-13T12:17:43.173364Z"
    }
   },
   "outputs": [],
   "source": [
    "test.loc[test.DT_W == 78 ,'D14_revised'] = test.loc[test.DT_W == 78 ,'D14_revised'].map(lambda x: np.nan if pd.isna(x) else x/900*530)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T12:36:28.175474Z",
     "start_time": "2019-10-13T12:36:26.208512Z"
    }
   },
   "outputs": [],
   "source": [
    "### 对时间进行细分，周内第几天和当天时间小时\n",
    "train['dow'] = train['TransactionDT'].dt.dayofweek\n",
    "train['hour'] = train['TransactionDT'].dt.hour\n",
    "test['dow'] = test['TransactionDT'].dt.dayofweek\n",
    "test['hour'] = test['TransactionDT'].dt.hour\n",
    "# train['month'] = train['TransactionDT'].dt.month\n",
    "# test['month'] = test['TransactionDT'].dt.month\n",
    "train['email_domain_comp'] = (train['P_emaildomain'].values == train['R_emaildomain'].values).astype(int)\n",
    "test['email_domain_comp'] = (test['P_emaildomain'].values == test['R_emaildomain'].values).astype(int)\n",
    "train.drop(['D9'],axis=1,inplace=True)\n",
    "test.drop(['D9'],axis=1,inplace=True)\n",
    "# X_train = train.drop(['TransactionID','TransactionDT'],axis=1)\n",
    "# X_test = test.drop(['TransactionID','TransactionDT'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T12:33:48.82293Z",
     "start_time": "2019-10-13T12:33:48.81917Z"
    }
   },
   "outputs": [],
   "source": [
    "#类别变量，需要进行LabelEncoder\n",
    "cat_columns = ['uid1','id_12', 'id_13', 'id_14', 'id_15', 'id_16', 'id_17', 'id_18', 'id_19', 'id_20', 'id_21', 'id_22', 'id_23', 'id_24', 'id_25', 'id_26', 'id_27', 'id_28', 'id_29',\n",
    "            'id_30', 'id_31', 'id_32', 'id_33', 'id_34', 'id_35', 'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo', 'ProductCD', 'card4', 'card6', 'M4','P_emaildomain',\n",
    "            'R_emaildomain', 'card1', 'card2', 'card3',  'card5', 'addr1', 'addr2', 'M1', 'M2', 'M3', 'M5', 'M6', 'M7', 'M8', 'M9','hour','dow','device_name', 'device_version', 'OS_id_30',  'browser_id_31']\n",
    "#进行count encoding的\n",
    "count_columns = ['uid1','id_13','id_14','id_17','id_18','id_19','id_20','id_21',\n",
    "                 'id_22','id_24','id_25','id_26','id_30','id_31','id_33',\n",
    "                 'DeviceInfo','card6','P_emaildomain','R_emaildomain','card1',\n",
    "                 'card2','card3','card5','addr1','addr2','hour','device_version','OS_id_30','browser_id_31']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "check1\n"
     ]
    }
   ],
   "source": [
    "print('check1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T12:38:23.760987Z",
     "start_time": "2019-10-13T12:36:41.988764Z"
    }
   },
   "outputs": [],
   "source": [
    "for f in cat_columns:\n",
    "    #if X_train[f].dtype=='object' or X_test[f].dtype=='object': \n",
    "    lbl = preprocessing.LabelEncoder()\n",
    "    lbl.fit(list(train[f].astype(str)) + list(test[f].astype(str)))\n",
    "    train[f] = lbl.transform(list(train[f].astype(str)))\n",
    "    test[f] = lbl.transform(list(test[f].astype(str))) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T12:38:25.296314Z",
     "start_time": "2019-10-13T12:38:23.762843Z"
    }
   },
   "outputs": [],
   "source": [
    "train.fillna(-999,inplace = True)\n",
    "test.fillna(-999,inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T12:38:32.513662Z",
     "start_time": "2019-10-13T12:38:31.628365Z"
    }
   },
   "outputs": [],
   "source": [
    "# 计数编码\n",
    "for i in count_columns:\n",
    "    train[i+'_count_full'] = train[i].map(pd.concat([train[i], test[i]], ignore_index=True).value_counts(dropna=False))\n",
    "    test[i+'_count_full'] = test[i].map(pd.concat([train[i], test[i]], ignore_index=True).value_counts(dropna=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "check1.5\n"
     ]
    }
   ],
   "source": [
    "print('check1.5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T12:44:22.143634Z",
     "start_time": "2019-10-13T12:44:15.603112Z"
    }
   },
   "outputs": [],
   "source": [
    "# 计算自然小时，自然天中的交易次数\n",
    "train_test_all = pd.concat([train[['TransactionDT','TransactionAmt']],test[['TransactionDT','TransactionAmt']]],ignore_index=True,sort=False)\n",
    "train_test_all['day_count'] = train_test_all.groupby(train_test_all.TransactionDT.dt.date)['TransactionAmt'].transform('count')\n",
    "train_test_all['hour_count'] = train_test_all.groupby(train_test_all.TransactionDT.map(lambda x:str(x)[:13]))['TransactionAmt'].transform('count')\n",
    "train['day_count'] = train_test_all[:train_len].day_count.tolist()\n",
    "test['day_count'] = train_test_all[train_len:].day_count.tolist()\n",
    "train['hour_count'] = train_test_all[:train_len].hour_count.tolist()\n",
    "test['hour_count'] = train_test_all[train_len:].hour_count.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T12:45:02.185798Z",
     "start_time": "2019-10-13T12:44:56.527931Z"
    }
   },
   "outputs": [],
   "source": [
    "# #计算自然小时，自然天中的交易总金额\n",
    "# train_test_all = pd.concat([train[['TransactionDT','TransactionAmt']],test[['TransactionDT','TransactionAmt']]],ignore_index=True,sort=False)\n",
    "# train_test_all['day_sum'] = train_test_all.groupby(train_test_all.TransactionDT.dt.date)['TransactionAmt'].transform('sum')\n",
    "# train_test_all['hour_sum'] = train_test_all.groupby(train_test_all.TransactionDT.map(lambda x:str(x)[:13]))['TransactionAmt'].transform('sum')\n",
    "# train['day_sum'] = train_test_all[:590540].day_sum.tolist()\n",
    "# test['day_sum'] = train_test_all[590540:].day_sum.tolist()\n",
    "# train['hour_sum'] = train_test_all[:590540].hour_sum.tolist()\n",
    "# test['hour_sum'] = train_test_all[590540:].hour_sum.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T12:45:13.526495Z",
     "start_time": "2019-10-13T12:45:05.158007Z"
    }
   },
   "outputs": [],
   "source": [
    "# #计算自然小时，自然天中的交易平均金额\n",
    "# train_test_all = pd.concat([train[['TransactionDT','TransactionAmt']],test[['TransactionDT','TransactionAmt']]],ignore_index=True,sort=False)\n",
    "# train_test_all['day_mean'] = train_test_all.groupby(train_test_all.TransactionDT.dt.date)['TransactionAmt'].transform('mean')\n",
    "# train_test_all['hour_mean'] = train_test_all.groupby(train_test_all.TransactionDT.map(lambda x:str(x)[:13]))['TransactionAmt'].transform('mean')\n",
    "# train['day_mean'] = train_test_all[:590540].day_mean.tolist()\n",
    "# test['day_mean'] = train_test_all[590540:].day_mean.tolist()\n",
    "# train['hour_mean'] = train_test_all[:590540].hour_mean.tolist()\n",
    "# test['hour_mean'] = train_test_all[590540:].hour_mean.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T12:52:41.822294Z",
     "start_time": "2019-10-13T12:52:34.205517Z"
    }
   },
   "outputs": [],
   "source": [
    "### 按照价格个类别确定商品id\n",
    "temp123 = ['TransactionAmt__ProductCD']\n",
    "for feature in temp123:\n",
    "    f1, f2 = feature.split('__')\n",
    "    train[feature] = train[f1].astype(str) + '_' + train[f2].astype(str)\n",
    "    test[feature] = test[f1].astype(str) + '_' + test[f2].astype(str)\n",
    "    le = LabelEncoder()\n",
    "    le.fit(list(train[feature].astype(str).values) + list(test[feature].astype(str).values))\n",
    "    train[feature] = le.transform(list(train[feature].astype(str).values))\n",
    "    test[feature] = le.transform(list(test[feature].astype(str).values))\n",
    "train.rename(columns = {'TransactionAmt__ProductCD':'ProductID'},inplace=True)\n",
    "test.rename(columns = {'TransactionAmt__ProductCD':'ProductID'},inplace=True)\n",
    "for i in ['ProductID']:\n",
    "    train[i+'_count_full'] = train[i].map(pd.concat([train[i], test[i]], ignore_index=True).value_counts(dropna=False))\n",
    "    test[i+'_count_full'] = test[i].map(pd.concat([train[i], test[i]], ignore_index=True).value_counts(dropna=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "check2\n"
     ]
    }
   ],
   "source": [
    "print('check2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T13:03:19.465258Z",
     "start_time": "2019-10-13T13:02:23.24184Z"
    }
   },
   "outputs": [],
   "source": [
    "###  类似的继续做一些交叉类别\n",
    "temp = ['DeviceInfo__P_emaildomain', \n",
    "        'card1__card5', \n",
    "        'card2__id_20',\n",
    "        'card5__P_emaildomain', \n",
    "        'addr1__card1',\n",
    "        'addr1__addr2',\n",
    "        'card1__card2',\n",
    "        'card2__addr1',\n",
    "        'card1__P_emaildomain',\n",
    "        'card2__P_emaildomain',\n",
    "        'addr1__P_emaildomain',\n",
    "        'DeviceInfo__id_31',\n",
    "        'DeviceInfo__id_20',\n",
    "        'DeviceType__id_31',\n",
    "        'DeviceType__id_20',\n",
    "        'DeviceType__P_emaildomain',\n",
    "        'card1__M4',\n",
    "        'card2__M4',\n",
    "        'addr1__M4',\n",
    "        'P_emaildomain__M4',\n",
    "       'uid1__ProductID',\n",
    "       'uid1__DeviceInfo']\n",
    "for feature in temp:\n",
    "    f1, f2 = feature.split('__')\n",
    "    train[feature] = train[f1].astype(str) + '_' + train[f2].astype(str)\n",
    "    test[feature] = test[f1].astype(str) + '_' + test[f2].astype(str)\n",
    "\n",
    "    le = LabelEncoder()\n",
    "    le.fit(list(train[feature].astype(str).values) + list(test[feature].astype(str).values))\n",
    "    train[feature] = le.transform(list(train[feature].astype(str).values))\n",
    "    test[feature] = le.transform(list(test[feature].astype(str).values))\n",
    "\n",
    "for i in temp:\n",
    "    train[i+'_count_full'] = train[i].map(pd.concat([train[i], test[i]], ignore_index=True).value_counts(dropna=False))\n",
    "    test[i+'_count_full'] = test[i].map(pd.concat([train[i], test[i]], ignore_index=True).value_counts(dropna=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "check3\n"
     ]
    }
   ],
   "source": [
    "print('check3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T13:08:27.357349Z",
     "start_time": "2019-10-13T13:06:41.870557Z"
    }
   },
   "outputs": [],
   "source": [
    "###  做一些类别和连续变量交叉\n",
    "con_fea = ['V258','C1','C14','C13','TransactionAmt','D15_revised','D2_revised','id_02','dist1','V294','C11']\n",
    "cat_fea = ['card1','card2','addr1','card4','R_emaildomain','P_emaildomain','ProductID','uid1']\n",
    "train_test = pd.concat([train[con_fea+cat_fea],test[con_fea+cat_fea]],ignore_index=True,sort=False)\n",
    "\n",
    "for cont in con_fea:\n",
    "    for cat in cat_fea:\n",
    "        train[cont+'_'+cat+'_mean'] = train_test[cont].map(lambda x:np.nan if x==-999 else x).groupby(train_test[cat]).transform('mean')[:train_len].tolist()\n",
    "        train[cont+'_'+cat+'_std'] = train_test[cont].map(lambda x:np.nan if x==-999 else x).groupby(train_test[cat]).transform('std')[:train_len].tolist()\n",
    "        test[cont+'_'+cat+'_mean'] = train_test[cont].map(lambda x:np.nan if x==-999 else x).groupby(train_test[cat]).transform('mean')[train_len:].tolist()\n",
    "        test[cont+'_'+cat+'_std'] =  train_test[cont].map(lambda x:np.nan if x==-999 else x).groupby(train_test[cat]).transform('std')[train_len:].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.fillna(-999,inplace=True)\n",
    "test.fillna(-999,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "check4\n"
     ]
    }
   ],
   "source": [
    "print('check4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del df\n",
    "del train_test\n",
    "del train_test_all\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.drop(['DeviceInfo','device_version','DT_D','DT_W','DT_M','D15',\n",
    "              'D2','D1','D4','D6','D10','D11','D12','D3','D5','D7','D8','D13','D14','TransactionAmt_ProductID_mean'],axis=1,inplace=True)\n",
    "test.drop(['DeviceInfo','device_version','DT_D','DT_W','DT_M','D15',\n",
    "             'D2','D1','D4','D6','D10','D11','D12','D3','D5','D7','D8','D13','D14','TransactionAmt_ProductID_mean'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "### 除掉之前350之后的特征,根据特征重要性排序\n",
    "# orders = pd.read_csv('importance.csv')\n",
    "# drop = orders.loc[350:,'Unnamed: 0'].tolist()\n",
    "drop = ['V256', 'V223', 'V19', 'V244', 'V324', 'V37', 'V200', 'card3', 'C1_P_emaildomain_mean', 'V131', 'V35', 'id_17_count_full', 'id_17', 'V30', 'V163', 'V81', 'V332', 'V164', 'D7_revised', 'id_02_R_emaildomain_std', 'V336', 'M9', 'V301', 'V251', 'M8', 'V275', 'V5', 'V272', 'V331', 'V215', 'V291', 'V129', 'C11_P_emaildomain_mean', 'id_34', 'V160', 'V139', 'V124', 'V159', 'V274', 'V59', 'V126', 'V52', 'V79', 'V271', 'V24', 'V137', 'V286', 'TransactionAmt_R_emaildomain_std', 'V335', 'V115', 'V198', 'V234', 'V298', 'V43', 'V258_card4_mean', 'V264', 'OS_id_30_count_full', 'V267', 'V169', 'V217', 'C3', 'V23', 'V287', 'id_18_count_full', 'V96', 'V208', 'card4', 'id_32', 'V232', 'V188', 'V4', 'V7', 'id_38', 'id_02_R_emaildomain_mean', 'V293', 'V219', 'V259', 'V276', 'C1_R_emaildomain_mean', 'V279', 'V102', 'V253', 'C13_R_emaildomain_std', 'V245', 'V73', 'browser_id_31', 'V74', 'V322', 'V209', 'V203', 'V273', 'V221', 'V40', 'V242', 'V289', 'D15_revised_R_emaildomain_std', 'C14_R_emaildomain_std', 'V150', 'V316', 'V239', 'V265', 'V278', 'V166', 'V172', 'V132', 'V93', 'V58', 'C1_R_emaildomain_std', 'V29', 'V300', 'V134', 'V254', 'V145', 'V141', 'C11_P_emaildomain_std', 'V292', 'V210', 'V231', 'V280', 'V158', 'V123', 'V135', 'V220', 'V39', 'V26', 'V238', 'id_11', 'V319', 'V125', 'id_37', 'V206', 'C1_card4_mean', 'V94', 'V304', 'V57', 'V270', 'V33', 'V170', 'V202', 'V218', 'V108', 'V303', 'V213', 'V222', 'V64', 'V263', 'V326', 'V10', 'V147', 'V101', 'V142', 'V97', 'V214', 'V105', 'V60', 'V171', 'V329', 'ProductCD', 'V216', 'V34', 'V25', 'V6', 'TransactionAmt_card4_mean', 'V212', 'V250', 'V3', 'V63', 'V194', 'id_36', 'V178', 'V42', 'V85', 'V193', 'V290', 'id_23', 'V258_card4_std', 'V15', 'V288', 'id_15', 'V182', 'V2', 'V192', 'V260', 'V235', 'id_26_count_full', 'V138', 'id_24', 'id_10', 'C1_card4_std', 'V11', 'id_08', 'id_25_count_full', 'id_07', 'V167', 'V51', 'V229', 'V248', 'V197', 'V230', 'V144', 'V233', 'V157', 'dist1_card4_std', 'V284', 'V140', 'addr2_count_full', 'V154', 'V22', 'V204', 'M1', 'V71', 'V211', 'V255', 'V72', 'TransactionAmt_card4_std', 'V1', 'V80', 'V184', 'V299', 'C11_R_emaildomain_mean', 'V173', 'V177', 'id_04', 'D15_revised_card4_std', 'V180', 'V228', 'V151', 'V186', 'OS_id_30', 'V109', 'DeviceType', 'V18', 'V17', 'id_26', 'V247', 'V9', 'V191', 'V148', 'V65', 'V196', 'id_21', 'V297', 'V46', 'V338', 'addr2', 'V95', 'V92', 'dist1_card4_mean', 'V334', 'V100', 'id_25', 'V179', 'V104', 'V116', 'V16', 'V183', 'id_21_count_full', 'V302', 'V199', 'V227', 'C11_R_emaildomain_std', 'V176', 'V249', 'V237', 'V327', 'id_16', 'V155', 'V8', 'V252', 'V175', 'V339', 'V330', 'V181', 'V190', 'C14_card4_mean', 'V14', 'V337', 'C14_card4_std', 'id_35', 'id_02_card4_mean', 'V110', 'id_12', 'V226', 'V168', 'V21', 'V153', 'V195', 'id_02_card4_std', 'V236', 'V174', 'id_28', 'V84', 'V32', 'V106', 'V41', 'V111', 'V112', 'V114', 'V146', 'V328', 'V50', 'id_29', 'C13_card4_mean', 'V103', 'V98', 'V121', 'id_24_count_full', 'D2_revised_card4_mean', 'had_id', 'V113', 'D2_revised_card4_std', 'V240', 'TransactionAmt_ProductID_std', 'V185', 'id_22_count_full', 'id_22', 'V31', 'C13_card4_std', 'V68', 'V88', 'V294_card4_std', 'V294_card4_mean', 'V122', 'dist1_R_emaildomain_mean', 'V118', 'V269', 'V107', 'V305', 'V117', 'V119', 'V120', 'C11_card4_mean', 'C11_card4_std', 'dist1_R_emaildomain_std', 'V89', 'V241', 'id_27', 'V325', 'V28', 'D15_revised_card4_mean', 'V27']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop1 = drop[:200]\n",
    "drop2 = drop[200:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T13:10:46.147752Z",
     "start_time": "2019-10-13T13:10:44.606687Z"
    }
   },
   "outputs": [],
   "source": [
    "train.drop(drop1,axis=1,inplace=True)\n",
    "test.drop(drop1,axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.drop(drop2,axis=1,inplace=True)\n",
    "test.drop(drop2,axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T13:08:37.813144Z",
     "start_time": "2019-10-13T13:08:31.451678Z"
    }
   },
   "outputs": [],
   "source": [
    "y_train = train['isFraud'].copy()\n",
    "X_train = train.drop(['TransactionID','isFraud','TransactionDT'],axis=1)\n",
    "X_test = test.drop(['TransactionID','TransactionDT'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "del train,test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T13:09:11.15861Z",
     "start_time": "2019-10-13T13:09:11.156163Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(590540, 359)\n",
      "(506691, 359)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T13:11:17.761374Z",
     "start_time": "2019-10-13T13:11:17.750114Z"
    }
   },
   "outputs": [],
   "source": [
    "cat = ['uid1','id_12', 'id_13', 'id_14', 'id_15', 'id_16', 'id_17', 'id_18', 'id_19', 'id_20', 'id_21', 'id_22', 'id_23', 'id_24', 'id_25', 'id_26', 'id_27', 'id_28', 'id_29',\n",
    "            'id_30', 'id_31', 'id_32', 'id_33', 'id_34', 'id_35', 'id_36', 'id_37', 'id_38', 'DeviceType', 'ProductCD', 'card4', 'card6', 'M4','P_emaildomain',\n",
    "            'R_emaildomain', 'card1', 'card2', 'card3',  'card5', 'addr1', 'addr2', 'M1', 'M2', 'M3', 'M5', 'M6', 'M7', 'M8', 'M9','hour','dow','device_name', 'OS_id_30',  'browser_id_31','ProductID',\n",
    "'DeviceInfo__P_emaildomain', \n",
    "        'card1__card5', \n",
    "        'card2__id_20',\n",
    "        'card5__P_emaildomain', \n",
    "        'addr1__card1',\n",
    "        'addr1__addr2',\n",
    "        'card1__card2',\n",
    "        'card2__addr1',\n",
    "        'card1__P_emaildomain',\n",
    "        'card2__P_emaildomain',\n",
    "        'addr1__P_emaildomain',\n",
    "        'DeviceInfo__id_31',\n",
    "        'DeviceInfo__id_20',\n",
    "        'DeviceType__id_31',\n",
    "        'DeviceType__id_20',\n",
    "        'DeviceType__P_emaildomain',\n",
    "        'card1__M4',\n",
    "        'card2__M4',\n",
    "        'addr1__M4',\n",
    "        'P_emaildomain__M4',\n",
    "       'uid1__ProductID',\n",
    "       'uid1__DeviceInfo']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T13:11:18.285746Z",
     "start_time": "2019-10-13T13:11:18.283561Z"
    }
   },
   "outputs": [],
   "source": [
    "for i in drop:\n",
    "    if i in cat:\n",
    "        cat.remove(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T13:11:37.683599Z",
     "start_time": "2019-10-13T13:11:19.075595Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# remove this call for catboost\n",
    "# for column in cat:\n",
    "#     train_set = set(X_train[column])\n",
    "#     test_set = set(X_test[column])\n",
    "#     tt = train_set.intersection(test_set)\n",
    "#     print('----------------------------------------')\n",
    "#     print(column)\n",
    "#     print(f'train:{len(tt)/len(train_set)}')\n",
    "#     print(f'test:{len(tt)/len(test_set)}')\n",
    "#     X_train[column] = X_train[column].map(lambda x: -999 if x not in tt else x)\n",
    "#     X_test[column] = X_test[column].map(lambda x: -999 if x not in tt else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T13:11:45.990477Z",
     "start_time": "2019-10-13T13:11:44.892415Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train.drop(['device_hash'],axis=1,inplace=True)\n",
    "X_test.drop(['device_hash'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T13:11:46.665041Z",
     "start_time": "2019-10-13T13:11:46.662209Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(590540, 358)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.to_pickle('./X_train2.pkl')\n",
    "X_test.to_pickle('./X_test2.pkl')\n",
    "y_train.to_frame().to_pickle('./y_train2.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T14:29:00.848557Z",
     "start_time": "2019-10-13T13:47:58.569723Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# kf=KFold(n_splits = 5)\n",
    "# resu1 = 0\n",
    "# impor1 = 0\n",
    "# y_pred = 0\n",
    "# stack_train = np.zeros([X_train.shape[0],])\n",
    "# for train_index, test_index in kf.split(X_train, y_train):\n",
    "#     X_train2= X_train.iloc[train_index,:]\n",
    "#     y_train2= y_train.iloc[train_index]\n",
    "#     X_test2= X_train.iloc[test_index,:]\n",
    "#     y_test2= y_train.iloc[test_index]\n",
    "#     clf = lgb.LGBMClassifier(n_estimators=10000, random_state=1995,subsample=0.7,\n",
    "#                              colsample_bytree=0.7,learning_rate=0.005,importance_type = 'gain',\n",
    "#                      max_depth = -1, num_leaves = 256,min_child_samples=20,min_split_gain = 0.001,\n",
    "#                        bagging_freq=1,reg_alpha = 0,reg_lambda = 0,n_jobs = -1,metric='None')\n",
    "#     clf.fit(X_train2,y_train2,eval_set = [(X_train2,y_train2),(X_test2,y_test2)], eval_metric = 'auc',early_stopping_rounds=500,verbose=100)\n",
    "#     temp_predict = clf.predict_proba(X_test2)[:,1]\n",
    "#     stack_train[test_index] = temp_predict\n",
    "#     y_pred += clf.predict_proba(X_test)[:,1]/5\n",
    "#     roc = roc_auc_score(y_test2, temp_predict)\n",
    "#     print(roc)\n",
    "#     resu1 += roc/5\n",
    "#     impor1 += clf.feature_importances_/5\n",
    "#     gc.collect()\n",
    "# print('End:',resu1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-13T14:29:16.589752Z",
     "start_time": "2019-10-13T14:29:13.659213Z"
    }
   },
   "outputs": [],
   "source": [
    "# resu = pd.read_csv('../input/ieee-fraud-detection/sample_submission.csv')\n",
    "# resu['isFraud'] = y_pred\n",
    "# resu.to_csv('lgb.csv',index=False)\n",
    "# a= pd.DataFrame()\n",
    "# a['train'] = stack_train\n",
    "# a.to_csv('lgb_train.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
